package com.zhht.checkcop.demo;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.log4j.Logger;

import com.zhht.checkcop.db.MysqlDb;
import com.zhht.checkcop.db.StaticDBMap;
import com.zhht.checkcop.parser.Parser88dusNovel;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class Update88dusNovel implements PageProcessor{
	private final static Logger log= Logger.getLogger(Update88dusNovel.class);
	private Boolean pageURL=new Boolean(true);
	
	private Site site = Site.me().setCycleRetryTimes(5).setSleepTime(1500).setTimeOut(6000);
	private String typeName;
	private int type;
	public Update88dusNovel(){}
	public Update88dusNovel(String typeName,int type){
		this.typeName = typeName;
		this.type = type;
	}

	public void process(Page page) {
    	try{
	    	List<String> sm=page.getHtml().xpath("//span[@class='sm']/a/b/text()").all();
	    	List<String> url=page.getHtml().xpath("//span[@class='sm']/a/@href").all();
	    	List<String> zj=page.getHtml().xpath("//span[@class='zj']/a/text()").all();
	    	List<String> zz=page.getHtml().xpath("//span[@class='zz']/text()").all();
	    	List<String> zs=page.getHtml().xpath("//span[@class='zs']/text()").all();
	    	List<String> sj=page.getHtml().xpath("//span[@class='sj']/text()").all();
	    	List<String> zt=page.getHtml().xpath("//span[@class='zt']/text()").all();
	    	List<String> fs=page.getHtml().xpath("//span[@class='fs']/text()").all();
	    	zz.remove(0);
	    	zs.remove(0);
	    	sj.remove(0);
	    	zt.remove(0);
	    	fs.remove(0);
	    	if(null != sm && sm.size()>0){
		    	page.putField("name", sm);
		    	page.putField("url", url);
		    	page.putField("zj", zj);
		    	page.putField("zz", zz);
		    	page.putField("zs", zs);
		    	page.putField("sj", sj);
		    	page.putField("zt", zt);
		    	page.putField("fs", fs);
		    	page.putField("typeName", typeName);
		    	
		    	String currentPage = page.getHtml().xpath("//div[@class='pagelink']/strong/text()").get();
		    	log.info("当前类型:"+typeName+"------------------------------currentPage:"+currentPage);
		    	if(pageURL){
		    		System.out.println("pageURL:"+pageURL);
		    		List<String> pageNum = page.getHtml().xpath("//div[@class='pagelink']/a/text()").all();
		    		if(pageNum!=null&&pageNum.size()>0){
		    			String urls = pageNum.get(pageNum.size()-1);
		    			page.addTargetRequests(getPageNum(Integer.parseInt(urls),type));
		    		}
		    		pageURL=false;
		    	}
	    	}
		} catch (Exception e) {
			e.printStackTrace();
		}
    }
    public List<String> getPageNum(Integer pageNum,Integer type){
    	List<String> page=new ArrayList<String>();
    	for(int i=2;i<=pageNum;i++){
    		page.add("https://www.88dus.com/sort"+type+"/"+i+"/");
    	}
    	return page;
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
    	try{
        	MysqlDb mysqldb = new MysqlDb();
        	mysqldb.connect();
//        	StaticDBMap.existNovelNameMap = mysqldb.query("select id,name,author_name from novel_info where source = 1 ",null,"name","author_name");
        	StaticDBMap.existNovelNameMap = mysqldb.query("select id,source_url from novel_info where source = 1 ",new ArrayList(),"source_url");
        	mysqldb.close();
    	}catch (Exception e) {
			e.printStackTrace();
		}

    	int thread=args!=null&&args.length>0?Integer.parseInt(args[0]):10;
	    StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("玄幻魔法",1)).addUrl("https://www.88dus.com/sort1/1/").addPipeline(new Parser88dusNovel("1","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("玄幻魔法已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("武侠修真",2)).addUrl("https://www.88dus.com/sort2/1/").addPipeline(new Parser88dusNovel("2","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("武侠修真已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("都市言情",3)).addUrl("https://www.88dus.com/sort3/1/").addPipeline(new Parser88dusNovel("3","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("都市言情已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("历史穿越",4)).addUrl("https://www.88dus.com/sort4/1/").addPipeline(new Parser88dusNovel("4","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("历史穿越已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("恐怖悬疑",5)).addUrl("https://www.88dus.com/sort5/1/").addPipeline(new Parser88dusNovel("5","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("恐怖悬疑已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("游戏竞技",6)).addUrl("https://www.88dus.com/sort6/1/").addPipeline(new Parser88dusNovel("6","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("游戏竞技已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("军事科幻",7)).addUrl("https://www.88dus.com/sort7/1/").addPipeline(new Parser88dusNovel("7","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("军事科幻已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("综合类型",8)).addUrl("https://www.88dus.com/sort8/1/").addPipeline(new Parser88dusNovel("8","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("综合类型已完成");
        StaticDBMap.writeNovelNameMap=new HashMap<String, Map<String,Object>>();
        StaticDBMap.updateExistNovelNameMap=new HashMap<String, Map<String,Object>>();
        Spider.create(new Update88dusNovel("女生频道",9)).addUrl("https://www.88dus.com/sort9/1/").addPipeline(new Parser88dusNovel("9","https://www.88dus.com")).thread(thread).run();
        saveAndUpdate();
        log.info("女生频道已完成");
    }
    
    
	private static void saveAndUpdate() {
		try{
        	MysqlDb mysqldb = new MysqlDb();
        	mysqldb.connect();
        	if(null != StaticDBMap.writeNovelNameMap && StaticDBMap.writeNovelNameMap.size() > 0){
	        	//新增
	        	Iterator<Entry<String, Map<String,Object>>> iter = StaticDBMap.writeNovelNameMap.entrySet().iterator();
	        	List<String[]> sqls=new ArrayList<String[]>();
	        	while (iter.hasNext()) {
	        		Map.Entry<String, Map<String,Object>> entry = iter.next();
	        		sqls.add(mysqldb.assemblyInsertSqls("novel_info",entry.getValue()));
	            }
	        	mysqldb.executeBatchs(sqls);
        	}
        	if(null != StaticDBMap.updateExistNovelNameMap && StaticDBMap.updateExistNovelNameMap.size() > 0){
        		//更新
	        	Iterator<Entry<String, Map<String,Object>>> iters = StaticDBMap.updateExistNovelNameMap.entrySet().iterator();
	        	List<String> sqls1=new ArrayList<String>();
	        	int i=0;
	        	while (iters.hasNext()) {
	        		Map.Entry<String, Map<String,Object>> entry = iters.next();
	        		
	        		Map<String,String[]> conditionMap=new HashMap<String,String[]>();
	        		String[] str=new String[2];
	        		str[0]="=";
	        		str[1]=entry.getKey();
	        		conditionMap.put("id", str);
	        		
	        		sqls1.add(mysqldb.assemblyUpdateSql("novel_info", entry.getValue(), conditionMap, null, false));
	        		if(i%100==0&&i!=0){
	        			mysqldb.executeBatch(sqls1);
	        			sqls1=new ArrayList<String>();
	        		}
	        		i++;
	            }
	        	if(i%100!=0){
	        		mysqldb.executeBatch(sqls1);
	        	}
        	}
        	
//        	try{
//        		//删除名称不一致但是同名的小说
//    			List<String> sqls=mysqldb.queryListStr("SELECT CONCAT('DELETE FROM novel_info where id = ',id) FROM novel_info tt INNER JOIN (SELECT SUBSTRING_INDEX(GROUP_CONCAT(create_date ORDER BY create_date DESC),',',1) as create_date,source_url FROM novel_info GROUP BY source_url HAVING count(1) > 1) aa on tt.source_url = aa.source_url and tt.create_date = aa.create_date");
//    			if(sqls!=null&&sqls.size()>0){
//    				mysqldb.executeBatch(sqls);
//    			}
//    		}catch(Exception e){
//    			e.printStackTrace();
//    		}
        	
        	mysqldb.close();
        }catch (Exception e) {
        	e.printStackTrace();
        }
	}
}
